# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import redis,re

class SpiderCsdncoursePipeline(object):
    def process_item(self, item, spider):
        print("="*20)
        return item

class MasterPipeline(object):
    def __init__(self,host,port):
        # 连接redis数据库
        self.r=redis.Redis(host=host,port=port,decode_responses=True)

    @classmethod
    def from_crawler(cls,crawler):
        return cls(
            host=crawler.settings.get("REDIS_HOST"),
            port=crawler.settings.get("REDIS_PORT")
        )
    def process_item(self,item,spider):
        # 判断url是否有效，并写入redis
        url=item["url"]
        if re.search('/course/detail/',url):
           self.r.lpush('coursespider:start_urls',url)
        else:
            self.r.lpush('coursespider:no_urls',url)